package com.main;

import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Vector;
import java.util.concurrent.CountDownLatch;

import com.dao.MYSQLControl;
import com.model.ProxyIp;
import com.service.ConnectThread;
import com.service.GetPageThread;
/*
 * 1.多线程获取页面抓取数据
 * 2.多线程校验获取的代理ip的可用性
 * 3.校验重复数据，插入数据库
 * 4.统计数据
 * 	
 */
public class ProxyIpMain {
	//爬的页面数
	private final static int count = 10;
	
	public static void main(String[] args) throws Exception{
		try{
			//1.多线程获取页面抓取数据
			Vector<ProxyIp> iplist = new Vector<ProxyIp>();
			CountDownLatch latch1 = new CountDownLatch(count);
			long ta = System.currentTimeMillis();
			for(int i = 1;i<= count;i++ ){
				String url="http://www.kuaidaili.com/free/inha/"+i;
				GetPageThread gpt = new GetPageThread(iplist,url,latch1,ta);
				gpt.start();
			}
			latch1.await();
			long tb = System.currentTimeMillis()-ta;
			System.out.println("请求页面,获取代理ip数:"+iplist.size()+", 页面获取成功率:"+(iplist.size()/15)+"/"+count+", 耗时："+formattime(tb));
			
			//2.多线程校验获取的代理ip的可用性
			CountDownLatch latch2 = new CountDownLatch(iplist.size());
			Vector<ProxyIp> avaliableIp = new Vector<ProxyIp>();
			long t1 = System.currentTimeMillis();
			for(int i = 0; i < iplist.size(); i++){
				ProxyIp proxyIp = iplist.get(i);
				ConnectThread ct = new ConnectThread(proxyIp,latch2,avaliableIp);
				ct.start();
			}
			latch2.await();
			long t2 = System.currentTimeMillis()-t1;
			System.out.println("爬虫校验完毕，代理ip数："+iplist.size()+",  可用:"+avaliableIp.size()+",  耗时:"+formattime(t2));
			
			//3.校验重复数据，插入数据库
			List<ProxyIp> insertList = new ArrayList<ProxyIp>();
			Map<String,String> singleip = new HashMap<String,String>();
			for(ProxyIp ip:avaliableIp){
				//保证插入的ip值唯一
				if(singleip.containsKey(ip.getIp())){
					continue;
				}else{
					singleip.put(ip.getIp(), ip.getPort());
				}
				//查看数据库中是否已经存在ip
				if(MYSQLControl.executeQuery(ip.getIp()) == null){
					insertList.add(ip);
				}
			}
			if(insertList.size() > 0){
				//将抓取的数据插入数据库
				MYSQLControl.executeInsert(insertList);
			}
			
			//4.统计数据
			int ava = avaliableIp.size();
			int input = singleip.size();
			int insert = insertList.size();
			System.out.println("统计--总共："+ava+", 本身重复："+(ava-input)+", 与数据库重复："+(input-insert)+", 插入："+insert);
			
		}catch(Exception e){
			e.printStackTrace();
		}
		
	}
	
	private static String formattime(long time){
		double td = time/1000.0;
        BigDecimal bg = new BigDecimal(td);
        td = bg.setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue();
		return td+"s";
	}

}
